package com.asiainfo.zqx;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import org.jsoup.select.Elements;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;

public class Shijibaodaowang extends BreadthCrawler {
    private String seedurl = "http://www.21jingji.com/";

    private String regurl = "http://www.21jingji.com/article/2022.*/herald/.*.html";

    public Shijibaodaowang(String crawlPath) {
        super(crawlPath, false);
        addSeed(seedurl);
        setThreads(1);
        //  setResumable(true);
    }

    @Override
    public void visit(Page page, CrawlDatums crawlDatums) {
        if (page.matchType("text") && page.select("div.main>h6").text().contains("21世纪经济报道")) {
            String url = page.url();
            String title = page.select("div.main>h1").text();
            //作者，时间，来源在同一行 例：2022年05月05日 10:32 21世纪经济报道 21财经APP 21世纪新健康研究院
            String tiwrsou = page.select("div.main>h6").text();
            String time = tiwrsou.substring(0, 17);
            //转为时间戳
            SimpleDateFormat formatter = new SimpleDateFormat("yyyy年MM月dd日 HH:mm", Locale.getDefault());
            //转为2022-1-1 11:11:11时间格式
            SimpleDateFormat righttime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault());
            Date dateString = null;
            try {
                dateString = formatter.parse(time);
                time = righttime.format(dateString);
            } catch (ParseException e) {
                e.printStackTrace();
            }
            String source = tiwrsou.substring(18, tiwrsou.lastIndexOf(" "));
            String writer = tiwrsou.substring(tiwrsou.lastIndexOf(" ")+1);
            String text = page.select("div.content>p").text();
            System.out.println("链接为" + url);
            System.out.println("标题" + title);
            System.out.println("时间为" + time);
            System.out.println("来源为" + source);
            System.out.println("作者为" + writer);
            System.out.println("正文为" + text);
        } else {
            Elements elements = page.select("a");
            for (int i = 0; i < elements.size(); i++) {
                if (elements.get(i).attr("abs:href").matches(regurl)) {
                    crawlDatums.add(new CrawlDatum(elements.get(i).attr("abs:href"), "text"));
                }
            }
        }
    }

    public static void main(String[] args) throws Exception {
        Shijibaodaowang shi = new Shijibaodaowang("shiji");
        shi.start(2);
    }
}
